Interpreting and explaining machine learning models
# packages for wrangling data and the original models
library(tidyverse)
library(tidymodels)
library(ranger)
library(glmnet)
library(kknn)
library(rcfss)
library(here) # defining consistent filepaths
# packages for model interpretation/explanation
library(DALEX)
library(DALEXtra)
# set random number generator seed value for reproducibility
set.seed(123)
theme_set(theme_minimal())Import models
# load Rdata file with all the data frames and pre-trained models
load(here("data", "models.Rdata"))
# show pre-trained models
rf_wf
## ══ Workflow [trained] ══════════════════════════════════════════════════════════
## Preprocessor: Recipe
## Model: rand_forest()
##
## ── Preprocessor ────────────────────────────────────────────────────────────────
## 4 Recipe Steps
##
## • step_novel()
## • step_impute_median()
## • step_impute_mode()
## • step_naomit()
##
## ── Model ───────────────────────────────────────────────────────────────────────
## Ranger result
##
## Call:
## ranger::ranger(x = maybe_data_frame(x), y = y, num.threads = 1, verbose = FALSE, seed = sample.int(10^5, 1))
##
## Type: Regression
## Number of trees: 500
## Sample size: 892
## Number of independent variables: 11
## Mtry: 3
## Target node size: 5
## Variable importance mode: none
## Splitrule: variance
## OOB prediction error (MSE): 8563494
## R squared (OOB): 0.445
glmnet_wf
## ══ Workflow [trained] ══════════════════════════════════════════════════════════
## Preprocessor: Recipe
## Model: linear_reg()
##
## ── Preprocessor ────────────────────────────────────────────────────────────────
## 7 Recipe Steps
##
## • step_novel()
## • step_impute_median()
## • step_impute_mode()
## • step_naomit()
## • step_dummy()
## • step_zv()
## • step_normalize()
##
## ── Model ───────────────────────────────────────────────────────────────────────
##
## Call: glmnet::glmnet(x = maybe_matrix(x), y = y, family = "gaussian", alpha = ~0.05)
##
## Df %Dev Lambda
## 1 0 0.0 39300
## 2 1 0.5 35800
## 3 2 1.1 32700
## 4 2 2.0 29800
## 5 4 2.9 27100
## 6 4 4.2 24700
## 7 6 5.7 22500
## 8 6 7.2 20500
## 9 7 8.7 18700
## 10 7 10.3 17000
## 11 7 11.8 15500
## 12 7 13.3 14100
## 13 7 14.8 12900
## 14 8 16.2 11700
## 15 8 17.6 10700
## 16 8 18.9 9740
## 17 9 20.2 8880
## 18 12 21.6 8090
## 19 13 23.1 7370
## 20 13 24.5 6720
## 21 13 25.8 6120
## 22 14 27.1 5580
## 23 19 28.4 5080
## 24 21 29.7 4630
## 25 23 31.0 4220
## 26 26 32.2 3840
## 27 27 33.4 3500
## 28 29 34.6 3190
## 29 32 35.6 2910
## 30 35 36.6 2650
## 31 37 37.7 2410
## 32 40 38.7 2200
## 33 42 39.6 2000
## 34 45 40.5 1830
## 35 48 41.3 1660
## 36 48 42.1 1520
## 37 49 42.8 1380
## 38 50 43.5 1260
## 39 52 44.1 1150
## 40 53 44.7 1040
## 41 54 45.2 952
## 42 55 45.7 867
## 43 56 46.1 790
## 44 58 46.5 720
## 45 58 46.9 656
## 46 58 47.2 598
##
## ...
## and 54 more lines.
kknn_wf
## ══ Workflow [trained] ══════════════════════════════════════════════════════════
## Preprocessor: Recipe
## Model: nearest_neighbor()
##
## ── Preprocessor ────────────────────────────────────────────────────────────────
## 7 Recipe Steps
##
## • step_novel()
## • step_impute_median()
## • step_impute_mode()
## • step_naomit()
## • step_dummy()
## • step_zv()
## • step_normalize()
##
## ── Model ───────────────────────────────────────────────────────────────────────
##
## Call:
## kknn::train.kknn(formula = ..y ~ ., data = data, ks = min_rows(10, data, 5))
##
## Type of response variable: continuous
## minimal mean absolute error: 2384
## Minimal mean squared error: 9884682
## Best kernel: optimal
## Best k: 10Create explainer objects
# use explain_*() to create explainer object
# first step of an DALEX operation
explainer_glmnet <- explain_tidymodels(
model = glmnet_wf,
# data should exclude the outcome feature
data = scorecard_train %>% select(-debt),
# y should be a vector containing the outcome of interest for the training set
y = scorecard_train$debt,
# assign a label to clearly identify model in later plots
label = "penalized regression"
)
## Preparation of a new explainer is initiated
## -> model label : penalized regression
## -> data : 892 rows 11 cols
## -> data : tibble converted into a data.frame
## -> target variable : 892 values
## -> predict function : yhat.workflow will be used ( [33m default [39m )
## -> predicted values : No value for predict function target column. ( [33m default [39m )
## -> model_info : package tidymodels , ver. 0.1.4 , task regression ( [33m default [39m )
## -> predicted values : numerical, min = 4549 , mean = 16564 , max = 23798
## -> residual function : difference between y and yhat ( [33m default [39m )
## -> residuals : numerical, min = -9383 , mean = -1.17e-11 , max = 8285
## [32m A new explainer has been created! [39m
explainer_rf <- explain_tidymodels(
model = rf_wf,
data = scorecard_train %>% select(-debt),
y = scorecard_train$debt,
label = "random forest"
)
## Preparation of a new explainer is initiated
## -> model label : random forest
## -> data : 892 rows 11 cols
## -> data : tibble converted into a data.frame
## -> target variable : 892 values
## -> predict function : yhat.workflow will be used ( [33m default [39m )
## -> predicted values : No value for predict function target column. ( [33m default [39m )
## -> model_info : package tidymodels , ver. 0.1.4 , task regression ( [33m default [39m )
## -> predicted values : numerical, min = 7167 , mean = 16577 , max = 24933
## -> residual function : difference between y and yhat ( [33m default [39m )
## -> residuals : numerical, min = -4550 , mean = -12.9 , max = 5052
## [32m A new explainer has been created! [39m
explainer_kknn <- explain_tidymodels(
model = kknn_wf,
data = scorecard_train %>% select(-debt),
y = scorecard_train$debt,
label = "k nearest neighbors"
)
## Preparation of a new explainer is initiated
## -> model label : k nearest neighbors
## -> data : 892 rows 11 cols
## -> data : tibble converted into a data.frame
## -> target variable : 892 values
## -> predict function : yhat.workflow will be used ( [33m default [39m )
## -> predicted values : No value for predict function target column. ( [33m default [39m )
## -> model_info : package tidymodels , ver. 0.1.4 , task regression ( [33m default [39m )
## -> predicted values : numerical, min = 8406 , mean = 16690 , max = 24768
## -> residual function : difference between y and yhat ( [33m default [39m )
## -> residuals : numerical, min = -6956 , mean = -126 , max = 8328
## [32m A new explainer has been created! [39mGlobal interpretation methods
Imputation-based feature importance
# random forest model first
vip_rf <- model_parts(explainer_rf)
plot(vip_rf)
# adjust sampling
## N = 100
model_parts(explainer_rf, N = 100) %>%
plot()
## all observations
model_parts(explainer_rf, N = NULL) %>%
plot()
# calculate ratio rather than raw change
model_parts(explainer_rf, type = "ratio") %>%
plot()
Exercises
- Calculate feature importance for the penalized regression and \(k\) nearest neighbors using all observations for permutations. How do they compare to the random forest model?
- Calculate feature importance for the random forest model three times, changing the random seed value before each calculation. How do the results change?
# compare to the glmnet model
vip_glmnet <- model_parts(explainer_glmnet, N = NULL)
plot(vip_glmnet)
# compare to the kknn model
vip_kknn <- model_parts(explainer_kknn, N = NULL)
plot(vip_kknn)
# calculate random forest feature importance thrice
set.seed(123)
model_parts(explainer_rf) %>% plot()
set.seed(234)
model_parts(explainer_rf) %>% plot()
set.seed(345)
model_parts(explainer_rf) %>% plot()
Partial dependence plots
# basic pdp for RF model and netcost variable
#
pdp_netcost <- model_profile(explainer_rf, variables = "netcost")
## just the PDP
plot(pdp_netcost)
## PDP with ICE curves
plot(pdp_netcost, geom = "profiles")
## larger sample size
model_profile(explainer_rf, variables = "netcost", N = 500) %>%
plot(geom = "profiles")
# group by type
pdp_cost_group <- model_profile(explainer_rf, variables = "netcost", groups = "type", N = NULL)
plot(pdp_cost_group, geom = "profiles")
Exercises
- Create PDP + ICE curves for netcost using all three models
- Create a PDP for all numeric variables in the penalized regression model
# create PDP + ICE curves for netcost from all three models
model_profile(explainer_rf, variables = "netcost", N = NULL) %>% plot(geom = "profiles")
model_profile(explainer_glmnet, variables = "netcost", N = NULL) %>% plot(geom = "profiles")
model_profile(explainer_kknn, variables = "netcost", N = NULL) %>% plot(geom = "profiles")
# create PDP for all numeric variables in glmnet model
model_profile(explainer_glmnet) %>%
plot()
- Create a PDP for the state variable and the random forest model
# PDP for state
## hard to read
pdp_state_kknn <- model_profile(explainer_kknn, variables = "state", N = NULL)
plot(pdp_state_kknn)
## manually construct and reorder states
## extract aggregated profiles
pdp_state_kknn$agr_profiles %>%
# convert to tibble
as_tibble() %>%
mutate(`_x_` = fct_reorder(.f = `_x_`, .x = `_yhat_`)) %>%
ggplot(mapping = aes(x = `_yhat_`, y = `_x_`, fill = `_yhat_`)) +
geom_col() +
scale_x_continuous(labels = scales::dollar) +
scale_fill_viridis_c(guide = "none") +
labs(
title = "Partial dependence plot for state",
subtitle = "Created for the k nearest neighbors model",
x = "Average prediction",
y = NULL
)
Local explanation methods
Choose a couple of observations to explain
# filter University of Chicago and Western Illinois University from original dataset
uchi <- filter(.data = scorecard, name == "University of Chicago") %>%
# remove unitid and name variables since they are not used in the model
select(-unitid, -name)
wiu <- filter(.data = scorecard, name == "Western Illinois University") %>%
select(-unitid, -name)Shapley values
# explain uchicago with rf model
shap_uchi_rf <- predict_parts(
explainer = explainer_rf,
new_observation = uchi,
type = "shap"
)
plot(shap_uchi_rf)
# explain uchicago with kknn model
shap_uchi_kknn <- predict_parts(
explainer = explainer_kknn,
new_observation = uchi,
type = "shap"
)
plot(shap_uchi_kknn)
# increase the number of feature order permutations
predict_parts(
explainer = explainer_kknn,
new_observation = uchi,
type = "shap",
B = 40
) %>%
plot()
Pair with ggplot2
# based on example from https://www.tmwr.org/explain.html#local-explanations
shap_uchi_kknn %>%
# convert to pure tibble-formatted data frame
as_tibble() %>%
# calculate average contribution per variable across permutations
group_by(variable) %>%
mutate(mean_val = mean(contribution)) %>%
ungroup() %>%
# reorder variable levels in order of absolute value of mean contribution
mutate(variable = fct_reorder(variable, abs(mean_val))) %>%
# define basic ggplot object for horizontal boxplot
ggplot(mapping = aes(x = contribution, y = variable, fill = mean_val > 0)) +
# add a bar plot
geom_col(
data = ~ distinct(., variable, mean_val),
mapping = aes(x = mean_val, y = variable),
alpha = 0.5
) +
# overlay with boxplot to show distribution
geom_boxplot(width = 0.5) +
# outcome variable is measured in dollars - contributions are the same units
scale_x_continuous(labels = scales::dollar) +
# use viridis color palette
scale_fill_viridis_d(guide = "none") +
labs(y = NULL)
Exercises
- Explain each model’s prediction for Western Illinois University. How do they differ?
# calculate shapley values
shap_wiu_rf <- predict_parts(
explainer = explainer_rf,
new_observation = wiu,
type = "shap"
)
shap_wiu_kknn <- predict_parts(
explainer = explainer_kknn,
new_observation = wiu,
type = "shap"
)
shap_wiu_glmnet <- predict_parts(
explainer = explainer_glmnet,
new_observation = wiu,
type = "shap"
)
# generate plots for each
plot(shap_wiu_rf)
plot(shap_wiu_kknn)
plot(shap_wiu_glmnet)
# view side by side
library(patchwork)
plot(shap_wiu_rf) +
plot(shap_wiu_kknn) +
plot(shap_wiu_glmnet)
# or combine together and reuse ggplot code from above
bind_rows(
shap_wiu_rf,
shap_wiu_kknn,
shap_wiu_glmnet
) %>%
# convert to pure tibble-formatted data frame
as_tibble() %>%
# calculate average contribution per variable across permutations
group_by(label, variable) %>%
mutate(mean_val = mean(contribution)) %>%
ungroup() %>%
# reorder variable levels in order of absolute value of mean contribution
# mutate(variable = fct_reorder(variable, abs(mean_val))) %>%
mutate(variable = tidytext::reorder_within(x = variable, by = abs(mean_val), within = label)) %>%
# define basic ggplot object for horizontal boxplot
ggplot(mapping = aes(x = contribution, y = variable, fill = mean_val > 0)) +
# add a bar plot
geom_col(
data = ~ distinct(., label, variable, mean_val),
mapping = aes(x = mean_val, y = variable),
alpha = 0.5
) +
# overlay with boxplot to show distribution
geom_boxplot(width = 0.5) +
# facet for each model
facet_wrap(vars(label), scales = "free_y") +
tidytext::scale_y_reordered() +
# outcome variable is measured in dollars - contributions are the same units
scale_x_continuous(labels = scales::dollar) +
# use viridis color palette
scale_fill_viridis_d(guide = "none") +
labs(y = NULL)
LIME
# load LIME package - note conflict with DALEX::explain()
library(lime)
# prepare the recipe
prepped_rec_rf <- extract_recipe(rf_wf)
# write a function to bake the observation
bake_rf <- function(x) {
bake(
prepped_rec_rf,
new_data = x
)
}
# create explainer object
lime_explainer_rf <- lime(
x = scorecard_train,
model = extract_fit_parsnip(rf_wf),
preprocess = bake_rf
)
# top 5 features
explanation_rf <- explain(
x = uchi,
explainer = lime_explainer_rf,
n_features = 5
)
plot_features(explanation_rf)
# top 10 features, increased permutations
explanation_rf <- explain(
x = uchi,
explainer = lime_explainer_rf,
n_features = 10,
n_permutations = 2000
)
plot_features(explanation_rf)
Exercises
- Calculate a LIME explanation for Western Illinois and the \(k\) nearest neighbors model. What are the top 10 features? How well does the local model explain the prediction?
- Reproduce the explanation but use a lasso model to select the most important features. How does the explanation change?
# prepare the recipe
prepped_rec_kknn <- extract_recipe(kknn_wf)
# write a function to bake the observation
bake_kknn <- function(x) {
bake(
prepped_rec_kknn,
new_data = x
)
}
# create explainer object
lime_explainer_kknn <- lime(
x = scorecard_train,
model = extract_fit_parsnip(kknn_wf),
preprocess = bake_kknn
)
# top 10 features
explanation_kknn <- explain(
x = uchi,
explainer = lime_explainer_kknn,
n_features = 10
)
plot_features(explanation_kknn)
# use lasso to select the most important features
explanation_lasso_kknn <- explain(
x = uchi,
explainer = lime_explainer_kknn,
n_features = 10,
feature_select = "lasso_path"
)
plot_features(explanation_lasso_kknn)
A note on the penalized regression model
Due to how the model was trained, bake_glmnet() requires an additional composition argument. Otherwise everything else is the same.
# prepare the recipe
prepped_rec_glmnet <- extract_recipe(glmnet_wf)
# write a function to convert the legislative description to an appropriate matrix object
bake_glmnet <- function(x) {
bake(
prepped_rec_glmnet,
new_data = x,
composition = "dgCMatrix"
)
}
# create explainer object
lime_explainer_glmnet <- lime(
x = scorecard_train,
model = extract_fit_parsnip(glmnet_wf),
preprocess = bake_glmnet
)
# top 5 features
explanation_glmnet <- explain(
x = uchi,
explainer = lime_explainer_glmnet,
n_features = 5)
plot_features(explanation_glmnet)
Session Info
sessioninfo::session_info()
## ─ Session info ───────────────────────────────────────────────────────────────
## setting value
## version R version 4.1.2 (2021-11-01)
## os macOS Monterey 12.1
## system aarch64, darwin20
## ui X11
## language (EN)
## collate en_US.UTF-8
## ctype en_US.UTF-8
## tz America/Chicago
## date 2022-02-15
## pandoc 2.14.2 @ /usr/local/bin/ (via rmarkdown)
##
## ─ Packages ───────────────────────────────────────────────────────────────────
## package * version date (UTC) lib source
## assertthat 0.2.1 2019-03-21 [1] CRAN (R 4.1.0)
## backports 1.4.1 2021-12-13 [1] CRAN (R 4.1.1)
## broom * 0.7.12 2022-01-28 [1] CRAN (R 4.1.1)
## bslib 0.3.1 2021-10-06 [1] CRAN (R 4.1.1)
## cellranger 1.1.0 2016-07-27 [1] CRAN (R 4.1.0)
## class 7.3-20 2022-01-13 [1] CRAN (R 4.1.1)
## cli 3.1.1 2022-01-20 [1] CRAN (R 4.1.1)
## codetools 0.2-18 2020-11-04 [1] CRAN (R 4.1.2)
## colorspace 2.0-2 2021-06-24 [1] CRAN (R 4.1.1)
## crayon 1.4.2 2021-10-29 [1] CRAN (R 4.1.1)
## DALEX * 2.3.0 2021-07-28 [1] CRAN (R 4.1.0)
## DALEXtra * 2.1.1 2021-05-09 [1] CRAN (R 4.1.0)
## DBI 1.1.2 2021-12-20 [1] CRAN (R 4.1.1)
## dbplyr 2.1.1 2021-04-06 [1] CRAN (R 4.1.0)
## dials * 0.0.10 2021-09-10 [1] CRAN (R 4.1.1)
## DiceDesign 1.9 2021-02-13 [1] CRAN (R 4.1.0)
## digest 0.6.29 2021-12-01 [1] CRAN (R 4.1.1)
## dplyr * 1.0.7 2021-06-18 [1] CRAN (R 4.1.0)
## ellipsis 0.3.2 2021-04-29 [1] CRAN (R 4.1.0)
## evaluate 0.14 2019-05-28 [1] CRAN (R 4.1.0)
## fansi 1.0.2 2022-01-14 [1] CRAN (R 4.1.1)
## farver 2.1.0 2021-02-28 [1] CRAN (R 4.1.0)
## fastmap 1.1.0 2021-01-25 [1] CRAN (R 4.1.0)
## forcats * 0.5.1 2021-01-27 [1] CRAN (R 4.1.1)
## foreach 1.5.1 2020-10-15 [1] CRAN (R 4.1.0)
## fs 1.5.2 2021-12-08 [1] CRAN (R 4.1.1)
## furrr 0.2.3 2021-06-25 [1] CRAN (R 4.1.0)
## future 1.23.0 2021-10-31 [1] CRAN (R 4.1.1)
## future.apply 1.8.1 2021-08-10 [1] CRAN (R 4.1.1)
## generics 0.1.2 2022-01-31 [1] CRAN (R 4.1.1)
## ggplot2 * 3.3.5 2021-06-25 [1] CRAN (R 4.1.1)
## glmnet * 4.1-3 2021-11-02 [1] CRAN (R 4.1.1)
## globals 0.14.0 2020-11-22 [1] CRAN (R 4.1.0)
## glue 1.6.1 2022-01-22 [1] CRAN (R 4.1.1)
## gower 0.2.2 2020-06-23 [1] CRAN (R 4.1.0)
## GPfit 1.0-8 2019-02-08 [1] CRAN (R 4.1.0)
## gtable 0.3.0 2019-03-25 [1] CRAN (R 4.1.1)
## hardhat 0.2.0 2022-01-24 [1] CRAN (R 4.1.1)
## haven 2.4.3 2021-08-04 [1] CRAN (R 4.1.1)
## here * 1.0.1 2020-12-13 [1] CRAN (R 4.1.0)
## highr 0.9 2021-04-16 [1] CRAN (R 4.1.0)
## hms 1.1.1 2021-09-26 [1] CRAN (R 4.1.1)
## htmltools 0.5.2 2021-08-25 [1] CRAN (R 4.1.1)
## httr 1.4.2 2020-07-20 [1] CRAN (R 4.1.0)
## iBreakDown 2.0.1 2021-05-07 [1] CRAN (R 4.1.1)
## igraph 1.2.11 2022-01-04 [1] CRAN (R 4.1.1)
## infer * 1.0.0 2021-08-13 [1] CRAN (R 4.1.1)
## ingredients 2.2.0 2021-04-10 [1] CRAN (R 4.1.1)
## ipred 0.9-12 2021-09-15 [1] CRAN (R 4.1.1)
## iterators 1.0.13 2020-10-15 [1] CRAN (R 4.1.0)
## janeaustenr 0.1.5 2017-06-10 [1] CRAN (R 4.1.0)
## jquerylib 0.1.4 2021-04-26 [1] CRAN (R 4.1.0)
## jsonlite 1.7.3 2022-01-17 [1] CRAN (R 4.1.1)
## kknn * 1.3.1 2016-03-26 [1] CRAN (R 4.1.0)
## knitr 1.37 2021-12-16 [1] CRAN (R 4.1.1)
## labeling 0.4.2 2020-10-20 [1] CRAN (R 4.1.0)
## lattice 0.20-45 2021-09-22 [1] CRAN (R 4.1.2)
## lava 1.6.10 2021-09-02 [1] CRAN (R 4.1.1)
## lhs 1.1.3 2021-09-08 [1] CRAN (R 4.1.1)
## lifecycle 1.0.1 2021-09-24 [1] CRAN (R 4.1.1)
## lime * 0.5.2 2021-02-24 [1] CRAN (R 4.1.1)
## listenv 0.8.0 2019-12-05 [1] CRAN (R 4.1.0)
## lubridate 1.8.0 2021-10-07 [1] CRAN (R 4.1.1)
## magrittr 2.0.2 2022-01-26 [1] CRAN (R 4.1.1)
## MASS 7.3-55 2022-01-13 [1] CRAN (R 4.1.1)
## Matrix * 1.4-0 2021-12-08 [1] CRAN (R 4.1.1)
## modeldata * 0.1.1 2021-07-14 [1] CRAN (R 4.1.0)
## modelr 0.1.8 2020-05-19 [1] CRAN (R 4.1.0)
## munsell 0.5.0 2018-06-12 [1] CRAN (R 4.1.0)
## nnet 7.3-17 2022-01-13 [1] CRAN (R 4.1.1)
## parallelly 1.30.0 2021-12-17 [1] CRAN (R 4.1.1)
## parsnip * 0.1.7 2021-07-21 [1] CRAN (R 4.1.0)
## patchwork * 1.1.1 2020-12-17 [1] CRAN (R 4.1.1)
## pillar 1.6.5 2022-01-25 [1] CRAN (R 4.1.2)
## pkgconfig 2.0.3 2019-09-22 [1] CRAN (R 4.1.0)
## plyr 1.8.6 2020-03-03 [1] CRAN (R 4.1.0)
## png 0.1-7 2013-12-03 [1] CRAN (R 4.1.0)
## pROC 1.18.0 2021-09-03 [1] CRAN (R 4.1.1)
## prodlim 2019.11.13 2019-11-17 [1] CRAN (R 4.1.0)
## purrr * 0.3.4 2020-04-17 [1] CRAN (R 4.1.0)
## R6 2.5.1 2021-08-19 [1] CRAN (R 4.1.1)
## ranger * 0.13.1 2021-07-14 [1] CRAN (R 4.1.0)
## rcfss * 0.2.3 2022-02-10 [1] local
## Rcpp 1.0.8 2022-01-13 [1] CRAN (R 4.1.1)
## readr * 2.1.1 2021-11-30 [1] CRAN (R 4.1.1)
## readxl 1.3.1 2019-03-13 [1] CRAN (R 4.1.0)
## recipes * 0.1.17 2021-09-27 [1] CRAN (R 4.1.1)
## reprex 2.0.1 2021-08-05 [1] CRAN (R 4.1.1)
## reticulate 1.24-9000 2022-02-09 [1] Github (rstudio/reticulate@273c98a)
## rlang 1.0.1 2022-02-03 [1] CRAN (R 4.1.1)
## rmarkdown 2.11 2021-09-14 [1] CRAN (R 4.1.1)
## rpart 4.1.16 2022-01-24 [1] CRAN (R 4.1.1)
## rprojroot 2.0.2 2020-11-15 [1] CRAN (R 4.1.0)
## rsample * 0.1.1 2021-11-08 [1] CRAN (R 4.1.1)
## rstudioapi 0.13 2020-11-12 [1] CRAN (R 4.1.0)
## rvest 1.0.2 2021-10-16 [1] CRAN (R 4.1.1)
## sass 0.4.0 2021-05-12 [1] CRAN (R 4.1.0)
## scales * 1.1.1 2020-05-11 [1] CRAN (R 4.1.0)
## sessioninfo 1.2.2 2021-12-06 [1] CRAN (R 4.1.1)
## shape 1.4.6 2021-05-19 [1] CRAN (R 4.1.0)
## SnowballC 0.7.0 2020-04-01 [1] CRAN (R 4.1.0)
## stringi 1.7.6 2021-11-29 [1] CRAN (R 4.1.1)
## stringr * 1.4.0 2019-02-10 [1] CRAN (R 4.1.1)
## survival 3.2-13 2021-08-24 [1] CRAN (R 4.1.2)
## tibble * 3.1.6 2021-11-07 [1] CRAN (R 4.1.1)
## tidymodels * 0.1.4 2021-10-01 [1] CRAN (R 4.1.1)
## tidyr * 1.1.4 2021-09-27 [1] CRAN (R 4.1.1)
## tidyselect 1.1.1 2021-04-30 [1] CRAN (R 4.1.0)
## tidytext 0.3.2 2021-09-30 [1] CRAN (R 4.1.1)
## tidyverse * 1.3.1 2021-04-15 [1] CRAN (R 4.1.0)
## timeDate 3043.102 2018-02-21 [1] CRAN (R 4.1.0)
## tokenizers 0.2.1 2018-03-29 [1] CRAN (R 4.1.0)
## tune * 0.1.6 2021-07-21 [1] CRAN (R 4.1.0)
## tzdb 0.2.0 2021-10-27 [1] CRAN (R 4.1.1)
## utf8 1.2.2 2021-07-24 [1] CRAN (R 4.1.0)
## vctrs 0.3.8 2021-04-29 [1] CRAN (R 4.1.0)
## viridisLite 0.4.0 2021-04-13 [1] CRAN (R 4.1.0)
## withr 2.4.3 2021-11-30 [1] CRAN (R 4.1.1)
## workflows * 0.2.4 2021-10-12 [1] CRAN (R 4.1.1)
## workflowsets * 0.1.0 2021-07-22 [1] CRAN (R 4.1.1)
## xfun 0.29 2021-12-14 [1] CRAN (R 4.1.1)
## xml2 1.3.3 2021-11-30 [1] CRAN (R 4.1.1)
## yaml 2.2.2 2022-01-25 [1] CRAN (R 4.1.1)
## yardstick * 0.0.9 2021-11-22 [1] CRAN (R 4.1.1)
##
## [1] /Library/Frameworks/R.framework/Versions/4.1-arm64/Resources/library
##
## ──────────────────────────────────────────────────────────────────────────────